"""
根据期刊名及url获取各期的期号,年份，以及每一期的期数量
"""
import random
import re
import traceback

import facade
import parsel
import pymysql
import requests
from xjlibrary.configread import MyConfigParser
from xjlibrary.myredis.myredisclient import getDataFromRedis
from xjlibrary.our_file_dir.base_dir import BaseDir

# 处理数据库语句中的转义
dealSqlTextFunc = lambda text: text.replace("'", "''") if isinstance(text, str) else ''

# 处理字典键值对
dealDictKVFunc = lambda key, dict_temp: dict_temp[key] if isinstance(dict_temp,
                                                                     dict) and key in dict_temp.keys() else ''

curPath = BaseDir.get_file_dir_absolute(__file__)
configfile = BaseDir.get_new_path(curPath, "db.ini")


class DownVolume(object):

    def __init__(self):
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(configfile, "db", logger=self.logger)
        self.req_error_log = BaseDir.get_new_path(curPath, "log", "reqArticleErrorLog.log")
        self.ini_dicts = MyConfigParser().set_config_path(
            configfile).get_configparser().set_keep_keys_case().read_config_string(
            'utf-8').conf_to_dict()
        self.USER_AGENT_LIST = [
            'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299'
        ]
        self.HEADERS = {
            'referer': 'https://www.tandfonline.com',
            # 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
            'user-agent': random.choice(self.USER_AGENT_LIST)
        }
        self.BASE_URL = 'https://www.tandfonline.com'
        self.BASE_VOLUME_URL = 'https://www.tandfonline.com/loi/{jid}'

    # 从数据库获取期刊
    def getTasksFromDB(self):
        """
        :param conn: 数据库连接对象
        :return: 查询结果
        """
        self.logger.info('Getting tasks from DB..')
        Sql = "SELECT `jid`, `name`,`volnum` FROM journal WHERE stat = 0 AND failcount < 3 ORDER BY RAND()"
        result = self.mysqlutils.SelectFromDB(Sql)
        return result

    def control(self):
        result = self.getTasksFromDB()
        if not result:
            self.logger.info('It is Empty from DB.')
            return
        self.startRunResult(result)

        self.logger.info('Get volume information completed.')

    def init_sn(self):
        self.session = requests.Session()  # 每次获取任务重新创建一个会话
        self.session.headers = self.HEADERS

    def startRunResult(self, result):
        """
        开始处理当前从数据库获取的任务
        :param session: 请求会话
        :param result: 数据库查询结果
        :return:
        """
        self.init_sn()
        if not isinstance(result, tuple):
            raise TypeError
        if not result:
            raise ValueError
        for journal_id, journal_name, volnumsql in result:

            journal_url = self.BASE_VOLUME_URL.format(jid=journal_id)

            journal_dict = self.getOneJournalInfo(journal_id, journal_url, journal_name)
            if journal_dict is not None:
                status = self.save2DBvolume(journal_dict)
                volnum = journal_dict['volume_num']
                self.update2DBjournal(journal_id, status, volnum)
            else:
                self.update2DBjournal(journal_id, False, 0)

    def getOneJournalInfo(self, journal_id, url, journal_name):
        """
        获取该期刊的信息
        :param session: 请求会话
        :param journal_id: 期刊id
        :param url: 期刊url
        :return: 该期刊的所有卷字典
        """
        self.logger.info('Start crawling jid:%s, url:%s' % (journal_id, url))
        one_journal_dict = {}
        one_journal_dict['jid'] = journal_id
        BoolResult, errString, r = facade.BaseRequest(url,
                                                      sn=self.session,
                                                      timeout=30,
                                                      headers=self.HEADERS,
                                                      verify=False)
        if BoolResult:
            one_journal_dict['volume_items'], volume_num = self.parseVolumeByHtml(r.text)
            one_journal_dict['volume_num'] = volume_num
            one_journal_dict['jname'] = journal_name
            return one_journal_dict
        else:
            return None

    # 将当前journal的所有卷保存至数据库
    def save2DBvolume(self, journal_dict):
        """
        将期刊卷保存至数据库
        :param conn: 数据库连接对象
        :param journal_dict: 期刊信息字典
        :return: None
        """
        ListSql = []
        try:
            if not isinstance(journal_dict, dict):
                raise TypeError
            if 'jid' not in journal_dict.keys() or not journal_dict['jid']:
                return None
            jid, jname = journal_dict['jid'], journal_dict["jname"]

            if 'volume_items' in journal_dict.keys() and journal_dict['volume_items']:
                for each_volume in journal_dict['volume_items']:
                    Sql = """replace INTO vol(volume, issue_url, jid, jname, `year`,issue,pages) VALUES('{volume}', '{issue_url}', '{jid}','{jname}','{year}','{issue}','{pages}')"""
                    vol, url, year, pages, issue = each_volume['volume'], each_volume['href'], each_volume[
                        'year'], each_volume['pages'], each_volume['issue']

                    Sql = Sql.format(volume=vol,
                                     issue_url=url, jid=jid,
                                     issue=issue, pages=pages,
                                     jname=pymysql.escape_string(jname), year=year)
                    ListSql.append(Sql)
            self.mysqlutils.ExeSqlListToDB(ListSql)
            return True

        except:
            print(traceback.format_exc())
            return False

    def update2DBjournal(self, jid, status, volnum):
        if status:
            Sql = "UPDATE journal SET stat=1,volnum=%s WHERE jid='%s' " % (volnum, jid)
        else:
            Sql = "UPDATE journal SET failcount=failcount + 1 WHERE jid='%s'" % jid
        self.mysqlutils.ExeSqlToDB(Sql)

    # 解析html文本,获取每一期的卷
    def parseVolumeByHtml(self, vol_html):
        """
        解析页面已
        :param vol_html: 卷信息页面
        :return: 卷信息列表
        """
        volume_list = []
        sel = parsel.Selector(vol_html)
        li_list = sel.xpath('//ul[@class="list-of-issues"]/li[@class="vol_li" and @id]')
        volume_num = len(li_list)
        for li in li_list:
            vols = li.xpath("./div/h3/text()").get().replace("Volume", "").strip()
            vol = re.sub('\d{4}', '', vols).strip().replace("-", "").strip()
            # 有可能卷就是年
            if vol == "":
                try:
                    vol = vols.split(" ")[1].strip()
                except:
                    pass
            issue_li = li.xpath("./ul/li")
            for issue_items in issue_li:
                one_volume_dict = {}
                url = issue_items.xpath("./a/@href").get().strip()
                year = issue_items.xpath('./a/p/span[@id="loiIssueCoverDateText"]/text()').get().strip()
                pages = issue_items.xpath('./a/p/span[@id="loiIssuePages"]/text()').get().strip()
                issue = issue_items.xpath('./a/div/text()').get().replace("Issue", "").strip()
                one_volume_dict['volume'] = vol
                one_volume_dict['href'] = url
                one_volume_dict['year'] = year
                one_volume_dict['pages'] = pages
                one_volume_dict['issue'] = issue
                volume_list.append(one_volume_dict)

        return volume_list, volume_num


    def get_proxy(self):
        return getDataFromRedis(curPath, 'db.ini')


def main():
    down = DownVolume()
    down.control()


if __name__ == '__main__':
    main()
